"""
    Topic：获取 douban 影评
    Version：1.0
    Date：2021-6-18
    Info：Tnterface + Chropath
"""
from selenium import webdriver
from lxml import etree
import time
import json
import csv


def GetJsonDate(driver,url):
    driver.get(url)
    driver.refresh()
    time.sleep(2)
    # 构建xpath元素树
    content = driver.page_source
    tree = etree.HTML(content)
    content_tree = tree.xpath('/html/body/pre/text()')
    # to json
    content_str = str(content_tree)
    content_str = content_str.replace("'","")
    content_list = json.loads(content_str)
    return content_list


#用于存储
# movies_detail_save = []
category_save = []
title_save = []
rate_save = []
url_save =[]
actors_save = []
short_comments_save = []
count =0

#分类
categories_url= 'https://movie.douban.com/j/search_tags?type=movie&tag=%E6%9C%80%E6%96%B0&source='
driver = webdriver.Chrome()
categories_list = GetJsonDate(driver,url=categories_url)
tags = categories_list[0]['tags']
#test
# tags = ['热门']
for tag in tags:
    print(tag)
    tag_url ='https://movie.douban.com/j/search_subjects?type=movie&tag='+ tag +'&page_limit=20&page_start=0'
    movies_list = GetJsonDate(driver,tag_url)
    #一类电影
    for movie_detail in movies_list[0]['subjects']:
        """
           片名：title
           评分：rate
           演员：type->list 需要转换
           评论：type->list 需要转换
           地址：url
                
           """
        # print(" 类别："+tag+" 片名："+movie_detail['title']+" 评分："+movie_detail['rate']+" 地址："+movie_detail['url'])
        movie_url = movie_detail['url']


        #一部电影
        driver.get(movie_url)
        time.sleep(2)
        content = driver.page_source
        tree = etree.HTML(content)
        actors = tree.xpath('//*[@id="info"]/span[3]/span[2]//a/text()')
        """
        主演：actors
        """
        #一部电影的影评
        short_comment_url = 'https://movie.douban.com/subject/'+movie_detail['id']+'/comments?status=P'
        driver.get(short_comment_url)
        time.sleep(2)
        content = driver.page_source
        tree = etree.HTML(content)
        short_comments = tree.xpath('//*[@id="comments"]//span[@class="short"]/text()')
        #类别 片名 评分 网址 主演 短评
        # print(" 类别：" + tag + " 片名：" + movie_detail['title'] + " 评分：" + movie_detail['rate']
        #       + " 地址：" + movie_detail['url'] + " 主演: " + str(actors)+" 短评： "+str(short_comments))

        # movies_detail_save.append(movie)
        category_save.append(tag)
        title_save.append(movie_detail['title'])
        rate_save.append(movie_detail['rate'])
        url_save.append(movie_detail['url'].replace("\\"," "))
        actors_save.append(str(actors).replace(","," "))
        short_comments_save.append(str(short_comments).replace(","," "))
        count = count +1





#save data to CSV
with open('res.csv',mode='w',encoding='gb18030')as file:
    # for movie in movies_detail_save:
    #     file.write(movie)

    #test
    filenames =["类别","片名","评分","网址","主演","短评"]
    f_csv = csv.DictWriter(file,fieldnames=filenames)
    f_csv.writeheader()
    for i in range(count):
       f_csv.writerow(
           {
               "类别":category_save[i],
               "片名":title_save[i],
               "评分":rate_save[i],
               "网址":url_save[i],
               "主演":actors_save[i],
               "短评":short_comments_save[i]

           }
       )

time.sleep(1)
driver.close()


